1   /*
2    * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
3    * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4    *
5    * This code is free software; you can redistribute it and/or modify it
6    * under the terms of the GNU General Public License version 2 only, as
7    * published by the Free Software Foundation.  Oracle designates this
8    * particular file as subject to the "Classpath" exception as provided
9    * by Oracle in the LICENSE file that accompanied this code.
10   *
11   * This code is distributed in the hope that it will be useful, but WITHOUT
12   * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13   * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
14   * version 2 for more details (a copy is included in the LICENSE file that
15   * accompanied this code).
16   *
17   * You should have received a copy of the GNU General Public License version
18   * 2 along with this work; if not, write to the Free Software Foundation,
19   * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20   *
21   * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22   * or visit www.oracle.com if you need additional information or have any
23   * questions.
24   */
25  
26  package build.tools.generatecharacter;
27  
28  import java.io.IOException;
29  import java.io.FileNotFoundException;
30  import java.io.BufferedReader;
31  import java.io.FileReader;
32  import java.io.PrintWriter;
33  import java.io.BufferedWriter;
34  import java.io.FileWriter;
35  import java.io.File;
36  import java.util.List;
37  
38  import build.tools.generatecharacter.CharacterName;
39  
40  /**
41   * This program generates the source code for the class java.lang.Character.
42   * It also generates native C code that can perform the same operations.
43   * It requires two external input data files:
44   * <ul>
45   * <li> Unicode specification file
46   * <li> Character class template file
47   * </ul>
48   * The Unicode specification file is available from the Unicode consortium.
49   * It has character specification lines that look like this:
50   * <listing>
51   * 0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061;
52   * </listing>
53   * The Character class template file is filled in with additional
54   * information to produce the file Character.java, which can then be
55   * compiled by a Java compiler.  The template file contains certain
56   * markers consisting of an alphabetic name string preceded by "$$".
57   * Such markers are replaced with generated program text.  As a special
58   * case, the marker "Lookup(xxx)" is recognized, where "xxx" consists of
59   * alphabetic characters constituting a variable name.  The character "_"
60   * is considered alphabetic for these purposes.
61   *
62   * @author  Guy Steele
63   * @author  Alan Liu
64   * @author  John O'Conner
65   */
66  
67  public class GenerateCharacter {
68  
69      final static boolean DEBUG = false;
70  
71      final static String commandMarker = "$$";
72      static String ROOT                        = "";
73      static String DefaultUnicodeSpecFileName  = ROOT + "UnicodeData.txt";
74      static String DefaultSpecialCasingFileName = ROOT + "SpecialCasing.txt";
75      static String DefaultPropListFileName     = ROOT + "PropList.txt";
76      static String DefaultJavaTemplateFileName = ROOT + "Character.java.template";
77      static String DefaultJavaOutputFileName   = ROOT + "Character.java";
78      static String DefaultCTemplateFileName    = ROOT + "Character.c.template";
79      static String DefaultCOutputFileName      = ROOT + "Character.c";
80  
81      static int plane = 0;
82  
83      /* The overall idea is that, in the generated Character class source code,
84      most character property data is stored in a special multi-level table whose
85      structure is defined by a sequence of nonnegative integers [k1, k2, ..., kn].
86      The integers must sum to 16 (the number of bits in a character).
87      The first table is indexed by the k1 high-order bits of the character code.
88      The result is concatenated to the next k2 bits of the character code to index
89      the second table, and so on.  Eventually the kn low-order bits of the character
90      code are concatenated and used to index one of two tables A and B; A contains
91      32-bit integer entries and B contains 16-bit short entries.  The 48 bits that
92      can be thus obtained encode the properties for the character.
93  
94      The default specification is [9, 4, 3, 0].  This particular table format was
95      designed by conducting an exhaustive search of table formats to minimize the
96      space consumed by the tables: the first and third tables need have only byte
97      values (the second table must have short values).  Another good choice is
98      [10, 6, 0], which produces a larger table but allows particularly fast table
99      lookup code.
100 
101     In each case, where the word "concatenated" is used, this may imply
102     first a << and then a | operation, or perhaps just a | operation if
103     the values in the table can be preshifted (generally possible if the table
104     entries are short rather than byte).
105     */
106 
107     /* The character properties are currently encoded into A (32 bits)and B (16 bits)
108        two parts.
109 
110     A: the low 32 bits are defined  in the following manner:
111 
112     1 bit Mirrored property.
113     4 bits      Bidirectional category (see below) (unused if -nobidi switch specified)
114     9 bits      A signed offset used for converting case .
115     1 bit       If 1, adding the signed offset converts the character to lowercase.
116     1 bit       If 1, subtracting the signed offset converts the character to uppercase.
117         Note: for a titlecase character, both of the preceding bits will be 1
118         and the signed offset will be 1.
119     1 bit   If 1, this character has a titlecase equivalent (possibly itself);
120         in this case, the two bits before this bit can be used to decide
121         whether this character is in fact uppercase, lowercase, or titlecase.
122     3 bits      This field provides a quick way to lex identifiers.
123         The eight possible values for this field are as follows:
124         0  May not be part of an identifier
125         1  Ignorable control; may continue a Unicode identifier or Java identifier
126         2  May continue a Java identifier but not a Unicode identifier (unused)
127         3  May continue a Unicode identifier or Java identifier
128         4  Is a Java whitespace character
129         5  May start or continue a Java identifier;
130            may continue but not start a Unicode identifier
131            (this value is used for connector punctuation such as _)
132         6  May start or continue a Java identifier;
133            may not occur in a Unicode identifier
134            (this value is used for currency symbols such as $)
135         7  May start or continue a Unicode identifier or Java identifier
136         Thus:
137            5, 6, 7 may start a Java identifier
138            1, 2, 3, 5, 6, 7 may continue a Java identifier
139            7 may start a Unicode identifier
140            1, 3, 5, 7 may continue a Unicode identifier
141            1 is ignorable within an identifier
142            4 is Java whitespace
143     2 bits      This field indicates whether the character has a numeric property.
144         The four possible values for this field are as follows:
145         0  This character has no numeric property.
146         1  Adding the digit offset to the character code and then
147            masking with 0x1F will produce the desired numeric value.
148         2  This character has a "strange" numeric value.
149         3  A Java supradecimal digit: adding the digit offset to the
150            character code, then masking with 0x1F, then adding 10
151            will produce the desired numeric value.
152     5 bits  The digit offset (see description of previous field)
153     5 bits      Character type (see below)
154 
155     B: the high 16 bits are defined as:
156     1 bit Other_Lowercase property
157     1 bit Other_Uppercase property
158     1 bit Other_Alphabetic property
159     1 bit Other_Math property
160     1 bit Ideographic property
161     1 bit Noncharacter codepoint property
162     */
163 
164 
165     // bit masks identify each component of a 32-bit property field described
166     // above.
167     // shift* indicates how many shifts right must happen to get the
168     // indicated property value in the lowest bits of the 32-bit space.
169     private static final int
170         shiftType           = 0,        maskType            =       0x001F,
171         shiftDigitOffset    = 5,        maskDigitOffset     =       0x03E0,
172         shiftNumericType    = 10,       maskNumericType     =       0x0C00,
173         shiftIdentifierInfo = 12,       maskIdentifierInfo  =       0x7000,
174                                         maskUnicodePart     =       0x1000,
175         shiftCaseInfo       = 15,       maskCaseInfo        =      0x38000,
176                                         maskLowerCase       =      0x20000,
177                                         maskUpperCase       =      0x10000,
178                                         maskTitleCase       =      0x08000,
179         shiftCaseOffset     = 18,       maskCaseOffset      =   0x07FC0000,
180         shiftCaseOffsetSign = 5,
181                                         // used only when calculating and
182                                         // storing digit offsets from char values
183                                         maskDigit               =   0x001F,
184                                         // case offset are 9 bits
185                                         maskCase                =   0x01FF,
186         shiftBidi           = 27,       maskBidi              = 0x78000000,
187         shiftMirrored       = 31,       //maskMirrored          = 0x80000000,
188         shiftPlane          = 16,       maskPlane = 0xFF0000;
189 
190     // maskMirrored needs to be long, if up 16-bit
191     private static final long maskMirrored          = 0x80000000L;
192 
193     // bit masks identify the 16-bit priperty field described above, in B
194     // table
195     private static final long
196         maskOtherLowercase  = 0x100000000L,
197         maskOtherUppercase  = 0x200000000L,
198         maskOtherAlphabetic = 0x400000000L,
199         maskOtherMath       = 0x800000000L,
200         maskIdeographic     = 0x1000000000L,
201         maskNoncharacterCP  = 0x2000000000L;
202 
203     // Can compare masked values with these to determine
204     // numeric or lexical types.
205     public static int
206         valueNotNumeric             = 0x0000,
207         valueDigit                  = 0x0400,
208         valueStrangeNumeric         = 0x0800,
209         valueJavaSupradecimal       = 0x0C00,
210         valueIgnorable              = 0x1000,
211         valueJavaOnlyPart           = 0x2000,
212         valueJavaUnicodePart        = 0x3000,
213         valueJavaWhitespace         = 0x4000,
214         valueJavaStartUnicodePart   = 0x5000,
215         valueJavaOnlyStart          = 0x6000,
216         valueJavaUnicodeStart       = 0x7000,
217         lowJavaStart                = 0x5000,
218         nonzeroJavaPart             = 0x3000,
219         valueUnicodeStart           = 0x7000;
220 
221     // these values are used when only identifier properties are generated
222     // for use in verifier code. Shortens the property down to a single byte.
223     private static final int
224         bitJavaStart            = 0x02,
225         bitJavaPart             = 0x01,
226         maskIsJavaIdentifierPart = bitJavaPart,
227         maskIsJavaIdentifierStart = bitJavaStart;
228 
229     static int maxOffset = maskCase/2 ;
230     static int minOffset = -maxOffset;
231 
232     /* The following routines provide simple, concise formatting of long integer values.
233      The number in the name of the method indicates the desired number of characters
234      to be produced.  If the number of digits required to represent the integer value
235      is less than that number, then the output is padded on the left  with zeros
236      (for hex) or with spaces (for decimal).  If the number of digits required to
237      represent the integer value is greater than the desired number, then all the digits
238      that are required are actually produced.
239     */
240 
241     static String hex(long n) { return Long.toHexString(n).toUpperCase(); }
242 
243     static String hex2(long n) {
244         String q = Long.toHexString(n & 0xFF).toUpperCase();
245         return "00".substring(Math.min(2, q.length())) + q;
246     }
247 
248     static String hex4(long n) {
249         String q = Long.toHexString(n & 0xFFFF).toUpperCase();
250         return "0000".substring(Math.min(4, q.length())) + q;
251     }
252 
253     static String hex8(long n) {
254         String q = Long.toHexString(n & 0xFFFFFFFFL).toUpperCase();
255         return "00000000".substring(Math.min(8, q.length())) + q;
256     }
257 
258     static String hex16(long n) {
259         String q = Long.toHexString(n).toUpperCase();
260         return "0000000000000000".substring(Math.min(16, q.length())) + q;
261     }
262 
263     static String dec3(long n) {
264         String q = Long.toString(n);
265         return "   ".substring(Math.min(3, q.length())) + q;
266     }
267 
268     static String dec5(long n) {
269         String q = Long.toString(n);
270         return "     ".substring(Math.min(5, q.length())) + q;
271     }
272 
273     /* This routine is called when some failure occurs. */
274 
275     static void FAIL(String s) {
276         System.out.println("** " + s);
277     }
278 
279     /**
280     * Given the data from the Unicode specification file, this routine builds a map.
281     *
282     * The specification file is assumed to contain its data in sorted order by
283     * character code; as a result, the array passed as an argument to this method
284     * has its components in the same sorted order, with one entry for each defined
285     * Unicode character or character range.  (A range is indicated by two consecutive
286     * entries, such that the name of the first entry begins with "<" and ends with
287     * "First>" and the second entry begins with "<" and ends with "Last>".)  This is
288     * therefore a sparse representation of the character property data.
289     *
290     * The resulting map is dense representation of the character data.  It contains
291     * 2^16 = 65536 entries, each of which is a long integer.  (Right now only 32 bits
292     * of this long value are used, but type long is used rather than int to facilitate
293     * future extensions of this source code generator that might require more than
294     * 32 bits to encode relevant character properties.)  Entry k holds the encoded
295     * properties for character k.
296     *
297     * Method buildMap manages the transformation from the sparse representation to
298     * the dense representation.  It calls method buildOne to handle the encoding
299     * of character property data from a single UnicodeSpec object into 32 bits.
300     * For undefined characters, method buildOne is not called and the map entry for
301     * that character is set to UnicodeSpec.UNASSIGNED.
302     *
303     * @param data       character property data from the Unicode specification file
304     * @return   an array of length 65536 with one entry for every possible char value
305     *
306     * @see GenerateCharacter#buildOne
307     */
308 
309     static long[] buildMap(UnicodeSpec[] data, SpecialCaseMap[] specialMaps, PropList propList)
310     {
311         long[] result;
312         if (bLatin1 == true) {
313             result = new long[256];
314         } else {
315             result = new long[1<<16];
316         }
317         int k=0;
318         int codePoint = plane<<16;
319         UnicodeSpec nonCharSpec = new UnicodeSpec();
320         for (int j = 0; j < data.length && k < result.length; j++) {
321             if (data[j].codePoint == codePoint) {
322                 result[k] = buildOne(codePoint, data[j], specialMaps);
323                 ++k;
324                 ++codePoint;
325             }
326             else if(data[j].codePoint > codePoint) {
327                 if (data[j].name.endsWith("Last>")) {
328                     // build map data for all chars except last in range
329                     while (codePoint < data[j].codePoint && k < result.length) {
330                         result[k] = buildOne(codePoint, data[j], specialMaps);
331                         ++k;
332                         ++codePoint;
333                     }
334                 }
335                 else {
336                     // we have a few unassigned chars before data[j].codePoint
337                     while (codePoint < data[j].codePoint && k < result.length) {
338                         result[k] = buildOne(codePoint, nonCharSpec, specialMaps);
339                         ++k;
340                         ++codePoint;
341                     }
342                 }
343                 k = data[j].codePoint & 0xFFFF;
344                 codePoint = data[j].codePoint;
345                 result[k] = buildOne(codePoint, data[j], specialMaps);
346                 ++k;
347                 ++codePoint;
348             }
349             else {
350                 System.out.println("An error has occured during spec mapping.");
351                 System.exit(0);
352             }
353         }
354         // if there are still unprocessed chars, process them
355         // as unassigned/undefined.
356         codePoint = (plane<<16) | k;
357         while (k < result.length) {
358             result[k] = buildOne(codePoint, nonCharSpec, specialMaps);
359             ++k;
360             ++codePoint;
361         }
362         // now add all extra supported properties from PropList, to the
363         // upper 16-bit
364         addExProp(result, propList, "Other_Lowercase", maskOtherLowercase);
365         addExProp(result, propList, "Other_Uppercase", maskOtherUppercase);
366         addExProp(result, propList, "Other_Alphabetic", maskOtherAlphabetic);
367         addExProp(result, propList, "Ideographic", maskIdeographic);
368         //addExProp(result, propList, "Other_Math", maskOtherMath);
369         //addExProp(result, propList, "Noncharacter_CodePoint", maskNoncharacterCP);
370 
371         return result;
372     }
373 
374     // The maximum and minimum offsets found while scanning the database
375     static int maxOffsetSeen = 0;
376     static int minOffsetSeen = 0;
377 
378     /**
379      * Some Unicode separator characters are not considered Java whitespace.
380      * @param c character to test
381      * @return true if c in an invalid Java whitespace character, false otherwise.
382      */
383     static boolean isInvalidJavaWhiteSpace(int c) {
384         int[] exceptions = {0x00A0, 0x2007, 0x202F, 0xFEFF};
385         boolean retValue = false;
386         for(int x=0;x<exceptions.length;x++) {
387             if(c == exceptions[x]) {
388                 retValue = true;
389                 break;
390             }
391         }
392         return retValue;
393 
394     }
395 
396     /**
397     * Given the character property data for one Unicode character, encode the data
398     * of interest into a single long integer value.  (Right now only 32 bits
399     * of this long value are used, but type long is used rather than int to facilitate
400     * future extensions of this source code generator that might require more than
401     * 32 bits to encode relevant character properties.)
402     *
403     * @param c   the character code for which to encode property data
404     * @param us  property data record from the Unicode specification file
405     *            (its character code might not be equal to c if it specifies data
406     *            for a range of characters)
407     * @return   an encoded long value that contains the properties for a single char
408     *
409     * @see GenerateCharacter#buildMap
410     */
411 
412     static long buildOne(int c, UnicodeSpec us, SpecialCaseMap[] specialMaps) {
413         long resultA = 0;
414         // record the general category
415         resultA |= us.generalCategory;
416 
417         // record the numeric properties
418         NUMERIC: {
419         STRANGE: {
420             int val = 0;
421             // c is A-Z
422             if ((c >= 0x0041) && (c <= 0x005A)) {
423                 val = c - 0x0041;
424                 resultA |= valueJavaSupradecimal;
425             // c is a-z
426             } else if ((c >= 0x0061) && (c <= 0x007A)) {
427                 val = c - 0x0061;
428                 resultA |= valueJavaSupradecimal;
429             // c is a full-width A-Z
430             } else if ((c >= 0xFF21) && (c <= 0xFF3A)) {
431                 val = c - 0xFF21;
432                 resultA |= valueJavaSupradecimal;
433             // c is a full-width a-z
434             } else if ((c >= 0xFF41) && (c <= 0xFF5A)) {
435                 val = c - 0xFF41;
436                 resultA |= valueJavaSupradecimal;
437             } else if (us.isDecimalValue()) {
438                 val = us.decimalValue;
439                 resultA |= valueDigit;
440             } else if (us.isDigitValue()) {
441                 val = us.digitValue;
442                 resultA |= valueDigit;
443             } else {
444                 if (us.numericValue.length() == 0) {
445                     break NUMERIC;                      // no numeric value at all
446                 } else {
447                     try {
448                         val = Integer.parseInt(us.numericValue);
449                         if (val >= 32 || val < 0) break STRANGE;
450                         if (c == 0x215F) break STRANGE;
451                     } catch(NumberFormatException e) {
452                         break STRANGE;
453                     }
454                     resultA |= valueDigit;
455                 }
456             }
457             if (val >= 32 || val < 0) break STRANGE;
458             resultA |= ((val - c & maskDigit) << shiftDigitOffset);
459             break NUMERIC;
460         } // end STRANGE
461         resultA |= valueStrangeNumeric;
462         } // end NUMERIC
463 
464         // record case mapping
465         int offset = 0;
466         // might have a 1:M mapping
467         int specialMap = SpecialCaseMap.find(c, specialCaseMaps);
468         boolean bHasUpper = (us.hasUpperMap()) || (specialMap != -1);
469         if (bHasUpper) {
470             resultA |= maskUpperCase;
471         }
472         if (specialMap != -1) {
473             // has mapping, but cannot record the
474             // proper offset; can only flag it and provide special case
475             // code in Character.java
476             offset = -1;
477         }
478         else if (us.hasUpperMap())  {
479             offset = c - us.upperMap;
480         }
481 
482         if (us.hasLowerMap()) {
483             resultA |= maskLowerCase;
484             if (offset == 0)
485                 offset = us.lowerMap - c;
486             else if (offset != (us.lowerMap - c)) {
487                 if (DEBUG) {
488                 FAIL("Character " + hex(c) +
489                 " has incompatible lowercase and uppercase mappings");
490                 }
491             }
492         }
493         if ((us.hasTitleMap() && us.titleMap != us.upperMap) ||
494             (bHasUpper && us.hasLowerMap())) {
495             resultA |= maskTitleCase;
496         }
497         if (bHasUpper && !us.hasLowerMap() && !us.hasTitleMap() && verbose) {
498             System.out.println("Warning: Character " + hex4(c) + " has upper but " +
499                                "no title case; Java won't know this");
500         }
501         if (offset < minOffsetSeen) minOffsetSeen = offset;
502         if (offset > maxOffsetSeen) maxOffsetSeen = offset;
503         if (offset > maxOffset || offset < minOffset) {
504             if (DEBUG) {
505             FAIL("Case offset " + offset + " for character " + hex4(c) + " must be handled as a special case");
506             }
507             offset = maskCase;
508         }
509         resultA |= ((offset & maskCase) << shiftCaseOffset);
510 
511         // record lexical info about this character
512         if (us.generalCategory == UnicodeSpec.LOWERCASE_LETTER
513                 || us.generalCategory == UnicodeSpec.UPPERCASE_LETTER
514                 || us.generalCategory == UnicodeSpec.TITLECASE_LETTER
515                 || us.generalCategory == UnicodeSpec.MODIFIER_LETTER
516                 || us.generalCategory == UnicodeSpec.OTHER_LETTER
517                 || us.generalCategory == UnicodeSpec.LETTER_NUMBER) {
518             resultA |= valueJavaUnicodeStart;
519         }
520         else if (us.generalCategory == UnicodeSpec.COMBINING_SPACING_MARK
521                 || us.generalCategory == UnicodeSpec.NON_SPACING_MARK
522                 || us.generalCategory == UnicodeSpec.DECIMAL_DIGIT_NUMBER) {
523             resultA |= valueJavaUnicodePart;
524         }
525         else if (us.generalCategory == UnicodeSpec.CONNECTOR_PUNCTUATION) {
526             resultA |= valueJavaStartUnicodePart;
527         }
528         else if (us.generalCategory == UnicodeSpec.CURRENCY_SYMBOL) {
529             resultA |= valueJavaOnlyStart;
530         }
531         else if (((c >= 0x0000) && (c <= 0x0008))
532                 || ((c >= 0x000E) && (c <= 0x001B))
533                 || ((c >= 0x007F) && (c <= 0x009F))
534                 || us.generalCategory == UnicodeSpec.FORMAT) {
535             resultA |= valueIgnorable;
536         }
537         else if (us.generalCategory == UnicodeSpec.SPACE_SEPARATOR
538                 || us.generalCategory == UnicodeSpec.LINE_SEPARATOR
539                 || us.generalCategory == UnicodeSpec.PARAGRAPH_SEPARATOR) {
540             if (!isInvalidJavaWhiteSpace(c)) resultA |= valueJavaWhitespace;
541         }
542         else if (((c >= 0x0009) && (c <= 0x000D))
543                 || ((c >= 0x001C) && (c <= 0x001F))) {
544             resultA |= valueJavaWhitespace;
545         }
546 
547         // record bidi category
548         if (!nobidi) {
549             int tmpBidi =
550                 (us.bidiCategory > UnicodeSpec.DIRECTIONALITY_OTHER_NEUTRALS ||
551                     us.bidiCategory == -1) ? maskBidi : (us.bidiCategory << shiftBidi);
552             resultA |= tmpBidi;
553         }
554 
555         // record mirrored property
556         if (!nomirror) {
557             resultA |= us.mirrored ? maskMirrored : 0;
558         }
559 
560         if (identifiers) {
561             long replacement = 0;
562             if ((resultA & maskIdentifierInfo) >= lowJavaStart) {
563                 replacement |= bitJavaStart;
564             }
565             if ( ((resultA & nonzeroJavaPart) != 0)
566                     && ((resultA & maskIdentifierInfo) != valueIgnorable)) {
567                 replacement |= bitJavaPart;
568             }
569             resultA = replacement;
570         }
571         return resultA;
572     }
573 
574     static void addExProp(long[] map, PropList propList, String prop, long mask) {
575         List<Integer> cps = propList.codepoints(prop);
576         if (cps != null) {
577             for (Integer cp : cps) {
578                 if (cp < map.length)
579                     map[cp] |= mask;
580             }
581         }
582     }
583 
584     /**
585     * This is the heart of the table compression strategy.  The inputs are a map
586     * and a number of bits (size).  The map is simply an array of long integer values;
587     * the number of bits indicates how index values for that map are to be split.
588     * The length of the given map must be a multiple of (1 << size).  The result is
589     * a new map z and a compressed table t such that for every valid index value k
590     * for the original map, t[(z[k>>size]<<size)|(k & ((1<<size)-1))] == map[k].
591     *
592     * In other words, the index k can be split into two parts, namely the "size"
593     * low-order bits and all the remaining high-order bits; the high-order bits are then
594     * remapped by map z to produce an index into table t.  In effect, the data of the
595     * original map m is broken up into blocks of size (1<<size); the compression relies
596     * on the expectation that many of these blocks will be identical and therefore need
597     * be represented only once in the compressed table t.
598     *
599     * This method is intended to be used iteratively.  The first map to be handed
600     * to it is the one constructed by method buildMap.  After that, the first of the
601     * two arrays returned by this method is fed back into it for further compression.
602     * At the end of the iteration, one has a starter map and a sequence of tables.
603     *
604     * The algorithm used to implement this computation is straightforward and not
605     * especially clever.  It uses brute-force linear search (the loop labeled MIDDLE)
606     * to locate identical blocks, so overall the time complexity of the algorithm
607     * is quadratic in the length of the input map.  Fortunately, speed is not crucial
608     * to this application.
609     *
610     * @param map                a map to be compressed
611     * @param size       the number of index bits to be split off by the compression
612     * @return   an array of length 2 containing two arrays; the first is a new map
613     *           and the second is a compressed data table
614     *
615     * @see GenerateCharacter#buildMap
616     */
617 
618     static long[][] buildTable(long[] map, int size) {
619         int n = map.length;
620         if (((n >> size) << size) != n) {
621             FAIL("Length " + n + " is not a multiple of " + (1 << size));
622         }
623         int m = 1 << size;
624         // We know the final length of the new map up front.
625         long[] newmap = new long[n >> size];
626         // The buffer is used temporarily to hold data for the compressed table
627         // because we don't know its final length yet.
628         long[] buffer = new long[n];
629         int ptr = 0;
630 OUTER:  for (int i = 0; i < n; i += m) {
631             // For every block of size m in the original map...
632     MIDDLE: for (int j = 0; j < ptr; j += m) {
633             // Find out whether there is already a block just like it in the buffer.
634                 for (int k = 0; k < m; k++) {
635                     if (buffer[j+k] != map[i+k])
636                         continue MIDDLE;
637                 }
638                 // There is a block just like it at position j, so just
639                 // put its index into the new map (thereby sharing it).
640                 newmap[i >> size] = (j >> size);
641                 continue OUTER;
642             } // end MIDDLE
643             // There is no block just like it already, so add it to
644             // the buffer and put its index into the new map.
645             for (int k = 0; k < m; k++) {
646                 buffer[ptr+k] = map[i+k];
647             }
648             newmap[i >> size] = (ptr >> size);
649             ptr += m;
650         } // end OUTER
651         // Now we know how long the compressed table should be,
652         // so create a new array and copy data from the temporary buffer.
653         long[] newdata = new long[ptr];
654         for (int j = 0; j < ptr; j++) {
655             newdata[j] = buffer[j];
656         }
657         // Return the new map and the new data table.
658         long[][] result = { newmap, newdata };
659         return result;
660     }
661 
662     /**
663     * Once the compressed tables have been computed, this method reads in a
664     * template file for the source code to be generated and writes out the final
665     * source code by acting as a sort of specialized macro processor.
666     *
667     * The first output line is a comment saying that the file was automatically
668     * generated; it includes a timestamp.  All other output is generated by
669     * reading a line from the template file, performing macro replacements,
670     * and then writing the resulting line or lines of code to the output file.
671     *
672     * This method handles the I/O, the timestamp comment, and the locating of
673     * macro calls within each input line.  The method replaceCommand is called
674     * to generate replacement text for each macro call.
675     *
676     * Macro calls to be replaced are indicated in the template file by
677     * occurrences of the commandMarker "$$".  The rest of the call may consist
678     * of Java letters (including the underscore "_") and also of balanced
679     * parentheses.
680     *
681     * @param theTemplateFileName
682     *           the file name for the template input file
683     * @param theOutputFileName
684     *           the file name for the source code output file
685     *
686     *     @see GenerateCharacter#replaceCommand
687     */
688 
689     static void generateCharacterClass(String theTemplateFileName,
690                                        String theOutputFileName)
691         throws FileNotFoundException, IOException {
692         BufferedReader in = new BufferedReader(new FileReader(theTemplateFileName));
693         PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(theOutputFileName)));
694         out.println(commentStart +
695             " This file was generated AUTOMATICALLY from a template file " +
696             new java.util.Date() + commentEnd);
697         int marklen = commandMarker.length();
698         LOOP: while(true) {
699             try {
700                 String line = in.readLine();
701                 if (line == null) break LOOP;
702                 int pos = 0;
703                 int depth = 0;
704                 while ((pos = line.indexOf(commandMarker, pos)) >= 0) {
705                     int newpos = pos + marklen;
706                     char ch = 'x';
707                     SCAN: while (newpos < line.length() &&
708                             (Character.isJavaIdentifierStart(ch = line.charAt(newpos))
709                             || ch == '(' || (ch == ')' && depth > 0))) {
710                         ++newpos;
711                         if (ch == '(') {
712                             ++depth;
713                         }
714                         else if (ch == ')') {
715                             --depth;
716                             if (depth == 0)
717                                 break SCAN;
718                         }
719                     }
720                     String replacement = replaceCommand(line.substring(pos + marklen, newpos));
721                     line = line.substring(0, pos) + replacement + line.substring(newpos);
722                     pos += replacement.length();
723                 }
724                 out.println(line);
725             }
726             catch (IOException e) {
727                 break LOOP;
728             }
729         }
730         in.close();
731         out.close();
732     }
733 
734     /**
735     * The replaceCommand method takes a command (a macro call without the
736     * leading marker "$$") and computes replacement text for it.
737     *
738     * Most of the commands are simply names of integer constants that are defined
739     * in the source code of this GenerateCharacter class.  The replacement text is
740     * simply the value of the constant as an appropriately formatted integer literal.
741     *
742     * Two cases are more complicated, however.  The command "Tables" causes the
743     * final map and compressed tables to be emitted, with elaborate comments
744     * describing their contents.  (This is actually handled by method genTables.)
745     * The command "Lookup(xxx)", where "xxx" is the name of a variable, generates
746     * an expression that will return the character property data for the character
747     * whose code is the value of the variable "xxx".  (this is handled by method
748     * "genAccess".)
749     *
750     * @param x  a command from the template file to be replaced
751     * @return   the replacement text, as a String
752     *
753     * @see GenerateCharacter#genTables
754     * @see GenerateCharacter#genAccess
755     * @see GenerateCharacter#generateCharacterClass
756     */
757 
758     static String replaceCommand(String x) {
759         if (x.equals("Tables")) return genTables();
760         if (x.equals("Initializers")) return genInitializers();
761         if (x.length() >= 9 && x.substring(0, 7).equals("Lookup(") &&
762                 x.substring(x.length()-1).equals(")") )
763             return genAccess("A", x.substring(7, x.length()-1), (identifiers ? 2 : 32));
764         if (x.length() >= 11 && x.substring(0, 9).equals("LookupEx(") &&
765                 x.substring(x.length()-1).equals(")") )
766             return genAccess("B", x.substring(9, x.length()-1), 16);
767         if (x.equals("shiftType")) return Long.toString(shiftType);
768         if (x.equals("shiftIdentifierInfo")) return Long.toString(shiftIdentifierInfo);
769         if (x.equals("maskIdentifierInfo")) return "0x" + hex8(maskIdentifierInfo);
770         if (x.equals("maskUnicodePart")) return "0x" + hex8(maskUnicodePart);
771         if (x.equals("shiftCaseOffset")) return Long.toString(shiftCaseOffset);
772         if (x.equals("shiftCaseInfo")) return Long.toString(shiftCaseInfo);
773         if (x.equals("shiftCaseOffsetSign")) return Long.toString(shiftCaseOffsetSign);
774         if (x.equals("maskCase")) return "0x" + hex8(maskCase);
775         if (x.equals("maskCaseOffset")) return "0x" + hex8(maskCaseOffset);
776         if (x.equals("maskLowerCase")) return "0x" + hex8(maskLowerCase);
777         if (x.equals("maskUpperCase")) return "0x" + hex8(maskUpperCase);
778         if (x.equals("maskTitleCase")) return "0x" + hex8(maskTitleCase);
779         if (x.equals("maskOtherLowercase")) return "0x" + hex4(maskOtherLowercase >> 32);
780         if (x.equals("maskOtherUppercase")) return "0x" + hex4(maskOtherUppercase >> 32);
781         if (x.equals("maskOtherAlphabetic")) return "0x" + hex4(maskOtherAlphabetic >> 32);
782         if (x.equals("maskIdeographic")) return "0x" + hex4(maskIdeographic >> 32);
783         if (x.equals("valueIgnorable")) return "0x" + hex8(valueIgnorable);
784         if (x.equals("valueJavaUnicodeStart")) return "0x" + hex8(valueJavaUnicodeStart);
785         if (x.equals("valueJavaOnlyStart")) return "0x" + hex8(valueJavaOnlyStart);
786         if (x.equals("valueJavaUnicodePart")) return "0x" + hex8(valueJavaUnicodePart);
787         if (x.equals("valueJavaOnlyPart")) return "0x" + hex8(valueJavaOnlyPart);
788         if (x.equals("valueJavaWhitespace")) return "0x" + hex8(valueJavaWhitespace);
789         if (x.equals("lowJavaStart")) return "0x" + hex8(lowJavaStart);
790         if (x.equals("nonzeroJavaPart")) return "0x" + hex8(nonzeroJavaPart);
791         if (x.equals("bitJavaStart")) return "0x" + hex8(bitJavaStart);
792         if (x.equals("bitJavaPart")) return Long.toString(bitJavaPart);
793         if (x.equals("valueUnicodeStart")) return "0x" + hex8(valueUnicodeStart);
794         if (x.equals("maskIsJavaIdentifierStart")) return "0x" + hex(maskIsJavaIdentifierStart);
795         if (x.equals("maskIsJavaIdentifierPart")) return "0x" + hex(maskIsJavaIdentifierPart);
796         if (x.equals("shiftDigitOffset")) return Long.toString(shiftDigitOffset);
797         if (x.equals("maskDigitOffset")) return "0x" + hex(maskDigitOffset);
798         if (x.equals("maskDigit")) return "0x" + hex(maskDigit);
799         if (x.equals("shiftNumericType")) return Long.toString(shiftNumericType);
800         if (x.equals("maskNumericType")) return "0x" + hex(maskNumericType);
801         if (x.equals("valueNotNumeric")) return "0x" + hex8(valueNotNumeric);
802         if (x.equals("valueDigit")) return "0x" + hex8(valueDigit);
803         if (x.equals("valueStrangeNumeric")) return "0x" + hex8(valueStrangeNumeric);
804         if (x.equals("valueJavaSupradecimal")) return "0x" + hex8(valueJavaSupradecimal);
805         if (x.equals("valueDigit")) return "0x" + hex8(valueDigit);
806         if (x.equals("valueStrangeNumeric")) return "0x" + hex8(valueStrangeNumeric);
807         if (x.equals("maskType")) return "0x" + hex(maskType);
808         if (x.equals("shiftBidi")) return Long.toString(shiftBidi);
809         if (x.equals("maskBidi")) return "0x" + hex(maskBidi);
810         if (x.equals("maskMirrored")) return "0x" + hex8(maskMirrored);
811         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.UNASSIGNED][UnicodeSpec.LONG]))
812             return Integer.toString(UnicodeSpec.UNASSIGNED);
813         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.UPPERCASE_LETTER][UnicodeSpec.LONG]))
814             return Integer.toString(UnicodeSpec.UPPERCASE_LETTER);
815         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.LOWERCASE_LETTER][UnicodeSpec.LONG]))
816             return Integer.toString(UnicodeSpec.LOWERCASE_LETTER);
817         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.TITLECASE_LETTER][UnicodeSpec.LONG]))
818             return Integer.toString(UnicodeSpec.TITLECASE_LETTER);
819         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.MODIFIER_LETTER][UnicodeSpec.LONG]))
820              return Integer.toString(UnicodeSpec.MODIFIER_LETTER);
821         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.OTHER_LETTER][UnicodeSpec.LONG]))
822              return Integer.toString(UnicodeSpec.OTHER_LETTER);
823         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.NON_SPACING_MARK][UnicodeSpec.LONG]))
824              return Integer.toString(UnicodeSpec.NON_SPACING_MARK);
825         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.ENCLOSING_MARK][UnicodeSpec.LONG]))
826              return Integer.toString(UnicodeSpec.ENCLOSING_MARK);
827         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.COMBINING_SPACING_MARK][UnicodeSpec.LONG]))
828              return Integer.toString(UnicodeSpec.COMBINING_SPACING_MARK);
829         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.DECIMAL_DIGIT_NUMBER][UnicodeSpec.LONG]))
830              return Integer.toString(UnicodeSpec.DECIMAL_DIGIT_NUMBER);
831         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.OTHER_NUMBER][UnicodeSpec.LONG]))
832              return Integer.toString(UnicodeSpec.OTHER_NUMBER);
833         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.SPACE_SEPARATOR][UnicodeSpec.LONG]))
834              return Integer.toString(UnicodeSpec.SPACE_SEPARATOR);
835         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.LINE_SEPARATOR][UnicodeSpec.LONG]))
836              return Integer.toString(UnicodeSpec.LINE_SEPARATOR);
837         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.PARAGRAPH_SEPARATOR][UnicodeSpec.LONG]))
838              return Integer.toString(UnicodeSpec.PARAGRAPH_SEPARATOR);
839         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.CONTROL][UnicodeSpec.LONG]))
840             return Integer.toString(UnicodeSpec.CONTROL);
841         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.FORMAT][UnicodeSpec.LONG]))
842             return Integer.toString(UnicodeSpec.FORMAT);
843         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.PRIVATE_USE][UnicodeSpec.LONG]))
844             return Integer.toString(UnicodeSpec.PRIVATE_USE);
845         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.SURROGATE][UnicodeSpec.LONG]))
846             return Integer.toString(UnicodeSpec.SURROGATE);
847         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.DASH_PUNCTUATION][UnicodeSpec.LONG]))
848             return Integer.toString(UnicodeSpec.DASH_PUNCTUATION);
849         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.START_PUNCTUATION][UnicodeSpec.LONG]))
850             return Integer.toString(UnicodeSpec.START_PUNCTUATION);
851         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.END_PUNCTUATION][UnicodeSpec.LONG]))
852             return Integer.toString(UnicodeSpec.END_PUNCTUATION);
853         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.INITIAL_QUOTE_PUNCTUATION][UnicodeSpec.LONG]))
854             return Integer.toString(UnicodeSpec.INITIAL_QUOTE_PUNCTUATION);
855         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.FINAL_QUOTE_PUNCTUATION][UnicodeSpec.LONG]))
856             return Integer.toString(UnicodeSpec.FINAL_QUOTE_PUNCTUATION);
857         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.CONNECTOR_PUNCTUATION][UnicodeSpec.LONG]))
858             return Integer.toString(UnicodeSpec.CONNECTOR_PUNCTUATION);
859         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.OTHER_PUNCTUATION][UnicodeSpec.LONG]))
860             return Integer.toString(UnicodeSpec.OTHER_PUNCTUATION);
861         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.LETTER_NUMBER][UnicodeSpec.LONG]))
862             return Integer.toString(UnicodeSpec.LETTER_NUMBER);
863         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.MATH_SYMBOL][UnicodeSpec.LONG]))
864             return Integer.toString(UnicodeSpec.MATH_SYMBOL);
865         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.CURRENCY_SYMBOL][UnicodeSpec.LONG]))
866             return Integer.toString(UnicodeSpec.CURRENCY_SYMBOL);
867         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.MODIFIER_SYMBOL][UnicodeSpec.LONG]))
868             return Integer.toString(UnicodeSpec.MODIFIER_SYMBOL);
869         if (x.equals(UnicodeSpec.generalCategoryList[UnicodeSpec.OTHER_SYMBOL][UnicodeSpec.LONG]))
870             return Integer.toString(UnicodeSpec.OTHER_SYMBOL);
871         if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT][UnicodeSpec.LONG]))
872             return Integer.toString(UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT);
873         if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT_EMBEDDING][UnicodeSpec.LONG]))
874             return Integer.toString(UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT_EMBEDDING);
875         if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT_OVERRIDE][UnicodeSpec.LONG]))
876             return Integer.toString(UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT_OVERRIDE);
877         if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT][UnicodeSpec.LONG]))
878             return Integer.toString(UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT);
879         if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC][UnicodeSpec.LONG]))
880             return Integer.toString(UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC);
881         if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_EMBEDDING][UnicodeSpec.LONG]))
882             return Integer.toString(UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_EMBEDDING);
883         if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_OVERRIDE][UnicodeSpec.LONG]))
884             return Integer.toString(UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT_OVERRIDE);
885         if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_POP_DIRECTIONAL_FORMAT][UnicodeSpec.LONG]))
886             return Integer.toString(UnicodeSpec.DIRECTIONALITY_POP_DIRECTIONAL_FORMAT);
887         if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER][UnicodeSpec.LONG]))
888             return Integer.toString(UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER);
889         if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR][UnicodeSpec.LONG]))
890             return Integer.toString(UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR);
891         if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER_TERMINATOR][UnicodeSpec.LONG]))
892             return Integer.toString(UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER_TERMINATOR);
893         if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_ARABIC_NUMBER][UnicodeSpec.LONG]))
894             return Integer.toString(UnicodeSpec.DIRECTIONALITY_ARABIC_NUMBER);
895         if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_COMMON_NUMBER_SEPARATOR][UnicodeSpec.LONG]))
896             return Integer.toString(UnicodeSpec.DIRECTIONALITY_COMMON_NUMBER_SEPARATOR);
897         if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_NONSPACING_MARK][UnicodeSpec.LONG]))
898             return Integer.toString(UnicodeSpec.DIRECTIONALITY_NONSPACING_MARK);
899          if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_BOUNDARY_NEUTRAL][UnicodeSpec.LONG]))
900             return Integer.toString(UnicodeSpec.DIRECTIONALITY_BOUNDARY_NEUTRAL);
901         if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_PARAGRAPH_SEPARATOR][UnicodeSpec.LONG]))
902             return Integer.toString(UnicodeSpec.DIRECTIONALITY_PARAGRAPH_SEPARATOR);
903         if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_SEGMENT_SEPARATOR][UnicodeSpec.LONG]))
904             return Integer.toString(UnicodeSpec.DIRECTIONALITY_SEGMENT_SEPARATOR);
905         if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_WHITESPACE][UnicodeSpec.LONG]))
906             return Integer.toString(UnicodeSpec.DIRECTIONALITY_WHITESPACE);
907         if (x.equals(UnicodeSpec.bidiCategoryList[UnicodeSpec.DIRECTIONALITY_OTHER_NEUTRALS][UnicodeSpec.LONG]))
908             return Integer.toString(UnicodeSpec.DIRECTIONALITY_OTHER_NEUTRALS);
909         FAIL("Unknown text substitution marker " + commandMarker + x);
910         return commandMarker + x;
911     }
912 
913     /**
914     * The genTables method generates source code for all the lookup tables
915     * needed to represent the various Unicode character properties.
916     * It simply calls the method genTable once for each table to be generated
917     * and then generates a summary comment.
918     *
919     * @return   the replacement text for the "Tables" command, as a String
920     *
921     * @see GenerateCharacter#genTable
922     * @see GenerateCharacter#replaceCommand
923     */
924     static String genTables() {
925         int n = sizes.length;
926         StringBuffer result = new StringBuffer();
927         // liu : Add a comment showing the source of this table
928         result.append(commentStart + " The following tables and code generated using:" +
929                   commentEnd + "\n  ");
930         result.append(commentStart + ' ' + commandLineDescription + commentEnd + "\n  ");
931 
932                 if (plane == 0 && bLatin1 == false) {
933             genCaseMapTableDeclaration(result);
934             genCaseMapTable(initializers, specialCaseMaps);
935                 }
936         int totalBytes = 0;
937         for (int k = 0; k < n - 1; k++) {
938             genTable(result, tableNames[k], tables[k], 0, bytes[k]<<3, sizes[k], preshifted[k],
939                 sizes[k+1], false, false, k==0);
940             int s = bytes[k];
941             if (s == 1 && useCharForByte) {
942                 s = 2;
943             }
944             totalBytes += tables[k].length * s;
945         }
946         genTable(result, "A", tables[n - 1], 0, (identifiers ? 2 : 32),
947             sizes[n - 1], false, 0, true, !(identifiers), false);
948 
949         // If we ever need more than 32 bits to represent the character properties,
950         // then a table "B" may be needed as well.
951         genTable(result, "B", tables[n - 1], 32, 16, sizes[n - 1], false, 0, true, true, false);
952 
953         totalBytes += ((((tables[n - 1].length * (identifiers ? 2 : 32)) + 31) >> 5) << 2);
954         result.append(commentStart);
955         result.append(" In all, the character property tables require ");
956         result.append(totalBytes).append(" bytes.").append(commentEnd);
957         if (verbose) {
958             System.out.println("The character property tables require "
959                  + totalBytes + " bytes.");
960         }
961         return result.toString();
962     }
963 
964     /**
965      * The genInitializers method generates the body of the
966      * ensureInitted() method, which enables lazy initialization of
967      * the case map table and other tables.
968      */
969     static String genInitializers() {
970         return initializers.toString();
971     }
972 
973     /**
974      * Return the total number of bytes needed by all tables.  This is a stripped-
975      * down copy of genTables().
976      */
977     static int getTotalBytes() {
978         int n = sizes.length;
979         int totalBytes = 0;
980         for (int k = 0; k < n - 1; k++) {
981             totalBytes += tables[k].length * bytes[k];
982         }
983         totalBytes += ((((tables[n - 1].length * (identifiers ? 2 : 32))
984                          + 31) >> 5) << 2);
985         return totalBytes;
986     }
987 
988     static void appendEscapedStringFragment(StringBuffer result,
989                                             char[] line,
990                                             int length,
991                                             boolean lastFragment) {
992         result.append("    \"");
993         for (int k=0; k<length; ++k) {
994             result.append("\\u");
995             result.append(hex4(line[k]));
996         }
997         result.append("\"");
998         result.append(lastFragment ? ";" : "+");
999         result.append("\n");
1000     }
1001 
1002     static String SMALL_INITIALIZER =
1003         "        { // THIS CODE WAS AUTOMATICALLY CREATED BY GenerateCharacter:\n"+
1004         // "            $$name = new $$type[$$size];\n"+
1005         "            int len = $$name_DATA.length();\n"+
1006         "            int j=0;\n"+
1007         "            for (int i=0; i<len; ++i) {\n"+
1008         "                int c = $$name_DATA.charAt(i);\n"+
1009         "                for (int k=0; k<$$entriesPerChar; ++k) {\n"+
1010         "                    $$name[j++] = ($$type)c;\n"+
1011         "                    c >>= $$bits;\n"+
1012         "                }\n"+
1013         "            }\n"+
1014         "            assert (j == $$size);\n"+
1015         "        }\n";
1016 
1017     static String SAME_SIZE_INITIALIZER =
1018         "        { // THIS CODE WAS AUTOMATICALLY CREATED BY GenerateCharacter:\n"+
1019         "            assert ($$name_DATA.length() == $$size);\n"+
1020         // "            $$name = new $$type[$$size];\n"+
1021         "            for (int i=0; i<$$size; ++i)\n"+
1022         "                $$name[i] = ($$type)$$name_DATA.charAt(i);\n"+
1023         "        }\n";
1024 
1025     static String BIG_INITIALIZER =
1026         "        { // THIS CODE WAS AUTOMATICALLY CREATED BY GenerateCharacter:\n"+
1027         // "            $$name = new $$type[$$size];\n"+
1028         "            int len = $$name_DATA.length();\n"+
1029         "            int j=0;\n"+
1030         "            int charsInEntry=0;\n"+
1031         "            $$type entry=0;\n"+
1032         "            for (int i=0; i<len; ++i) {\n"+
1033         "                entry |= $$name_DATA.charAt(i);\n"+
1034         "                if (++charsInEntry == $$charsPerEntry) {\n"+
1035         "                    $$name[j++] = entry;\n"+
1036         "                    entry = 0;\n"+
1037         "                    charsInEntry = 0;\n"+
1038         "                }\n"+
1039         "                else {\n"+
1040         "                    entry <<= 16;\n"+
1041         "                }\n"+
1042         "            }\n"+
1043         "            assert (j == $$size);\n"+
1044         "        }\n";
1045 
1046     static String INT32_INITIALIZER =
1047         "        { // THIS CODE WAS AUTOMATICALLY CREATED BY GenerateCharacter:\n"+
1048         "            char[] data = $$name_DATA.toCharArray();\n"+
1049         "            assert (data.length == ($$size * 2));\n"+
1050         "            int i = 0, j = 0;\n"+
1051         "            while (i < ($$size * 2)) {\n"+
1052         "                int entry = data[i++] << 16;\n"+
1053         "                $$name[j++] = entry | data[i++];\n"+
1054         "            }\n"+
1055         "        }\n";
1056 
1057     static void addInitializer(String name, String type, int entriesPerChar,
1058                                int bits, int size) {
1059 
1060         String template = (entriesPerChar == 1) ? SAME_SIZE_INITIALIZER :
1061                           ((entriesPerChar > 0) ? SMALL_INITIALIZER : BIG_INITIALIZER);
1062         if (entriesPerChar == -2) {
1063             template = INT32_INITIALIZER;
1064         }
1065         int marklen = commandMarker.length();
1066         int pos = 0;
1067         while ((pos = template.indexOf(commandMarker, pos)) >= 0) {
1068             int newpos = pos + marklen;
1069             char ch = 'x';
1070             while (newpos < template.length() &&
1071                    Character.isJavaIdentifierStart(ch = template.charAt(newpos)) &&
1072                    ch != '_') // Don't allow this in token names
1073                 ++newpos;
1074             String token = template.substring(pos+marklen, newpos);
1075             String replacement = "ERROR";
1076 
1077             if (token.equals("name")) replacement = name;
1078             else if (token.equals("type")) replacement = type;
1079             else if (token.equals("bits")) replacement = ""+bits;
1080             else if (token.equals("size")) replacement = ""+size;
1081             else if (token.equals("entriesPerChar")) replacement = ""+entriesPerChar;
1082             else if (token.equals("charsPerEntry")) replacement = ""+(-entriesPerChar);
1083             else FAIL("Unrecognized token: " + token);
1084 
1085             template = template.substring(0, pos) + replacement + template.substring(newpos);
1086             pos += replacement.length();
1087         }
1088         initializers.append(template);
1089     }
1090 
1091     /**
1092     * The genTable method generates source code for one lookup table.
1093     * Most of the complexity stems from handling various options as to
1094     * the type of the array components, the precise representation of the
1095     * values, the format in which to render each value, the number of values
1096     * to emit on each line of source code, and the kinds of useful comments
1097     * to be generated.
1098     *
1099     * @param result     a StringBuffer, to which the generated source code
1100     *                   text is to be appended
1101     * @param name       the name of the table
1102     * @param table      the table data (an array of long values)
1103     * @param extract    a distance, in bits, by which each entry of the table
1104     *                   is to be right-shifted before it is processed
1105     * @param bits       the number of bits (not bytes) to be used to represent
1106     *                   each table entry
1107     * @param size       the table data is divided up into blocks of size (1<<size);
1108     *                   in this method, this information is used only to affect
1109     *                   how many table values are to be generated per line
1110     * @param preshifted if this flag is true, then the table entries are to be
1111     *                   emitted in a preshifted form; that is, each value should
1112     *                   be left-shifted by the amount "shift", so that this work
1113     *                   is built into the table and need not be performed by an
1114     *                   explicit shift operator at run time
1115     * @param shift      this is the shift amount for preshifting of table entries
1116     * @param hexFormat  if this flag is true, table entries should be emitted as
1117     *                   hexadecimal literals; otherwise decimal literals are used
1118     * @param properties if this flag is true, the table entries are encoded
1119     *                   character properties rather than indexes into yet other tables;
1120     *                   therefore comments describing the encoded properties should
1121     *                   be generated
1122     * @param hexComment if this flag is true, each line of output is labelled with
1123     *                   a hexadecimal comment indicating the character values to
1124     *                   which that line applies; otherwise, decimal values indicating
1125     *                   table indices are generated
1126     *
1127     * @see GenerateCharacter#genTables
1128     * @see GenerateCharacter#replaceCommand
1129     */
1130 
1131     static void genTable(StringBuffer result, String name,
1132                          long[] table, int extract, int bits, int size,
1133                          boolean preshifted, int shift, boolean hexFormat,
1134                          boolean properties, boolean hexComment) {
1135 
1136         String atype = bits == 1 ? (Csyntax ? "unsigned long" : "int") :
1137             bits == 2 ? (Csyntax ? "unsigned long" : "int") :
1138             bits == 4 ? (Csyntax ? "unsigned long" : "int") :
1139             bits == 8 ? (Csyntax ? "unsigned char" : "byte") :
1140             bits == 16 ? (Csyntax ? "unsigned short" : "char") :
1141             bits == 32 ? (Csyntax ? "unsigned long" : "int") :
1142             (Csyntax ? "int64" : "long");
1143         long maxPosEntry = bits == 1 ? Integer.MAX_VALUE : // liu
1144             bits == 2 ? Integer.MAX_VALUE :
1145             bits == 4 ? Integer.MAX_VALUE :
1146             bits == 8 ? Byte.MAX_VALUE :
1147             bits == 16 ? Short.MAX_VALUE :
1148             bits == 32 ? Integer.MAX_VALUE :
1149             Long.MAX_VALUE;
1150         int entriesPerChar = bits <= 16 ? (16 / bits) : -(bits / 16);
1151         boolean shiftEntries = preshifted && shift != 0;
1152         if (bits == 8 && tableAsString && useCharForByte) {
1153             atype = "char";
1154             maxPosEntry = Character.MAX_VALUE;
1155             entriesPerChar = 1;
1156         }
1157         boolean noConversion = atype.equals("char");
1158 
1159         result.append(commentStart);
1160         result.append(" The ").append(name).append(" table has ").append(table.length);
1161         result.append(" entries for a total of ");
1162         int sizeOfTable = ((table.length * bits + 31) >> 5) << 2;
1163         if (bits == 8 && useCharForByte) {
1164             sizeOfTable *= 2;
1165         }
1166         result.append(sizeOfTable);
1167         result.append(" bytes.").append(commentEnd).append("\n\n");
1168         if (Csyntax)
1169             result.append("  static ");
1170         else
1171             result.append("  static final ");
1172         result.append(atype);
1173         result.append(" ").append(name).append("[");
1174         if (Csyntax)
1175             result.append(table.length >> (bits == 1 ? 5 : bits == 2 ? 4 : bits == 4 ? 3 : 0));
1176         if (tableAsString) {
1177             if (noConversion) {
1178                 result.append("] = (\n");
1179             } else {
1180                 result.append("] = new ").append(atype).append("["+table.length+"];\n  ");
1181                 result.append("static final String ").append(name).append("_DATA =\n");
1182             }
1183             int CHARS_PER_LINE = 8;
1184             StringBuffer theString = new StringBuffer();
1185             int entriesInCharSoFar = 0;
1186             char ch = '\u0000';
1187             int charsPerEntry = -entriesPerChar;
1188             for (int j=0; j<table.length; ++j) {
1189                 //long entry = table[j] >> extract;
1190                 long entry;
1191                 if ("A".equals(name))
1192                     entry = (table[j] & 0xffffffffL) >> extract;
1193                 else
1194                     entry = (table[j] >> extract);
1195                 if (shiftEntries) entry <<= shift;
1196                 if (entry >= (1L << bits)) {
1197                     FAIL("Entry too big");
1198                 }
1199                 if (entriesPerChar > 0) {
1200                     // Pack multiple entries into a character
1201                     ch = (char)(((int)ch >> bits) | (entry << (entriesPerChar-1)*bits));
1202                     ++entriesInCharSoFar;
1203                     if (entriesInCharSoFar == entriesPerChar) {
1204                         // Character is full
1205                         theString.append(ch);
1206                         entriesInCharSoFar = 0;
1207                         ch = '\u0000';
1208                     }
1209                 }
1210                 else {
1211                     // Use multiple characters per entry
1212                     for (int k=0; k<charsPerEntry; ++k) {
1213                         ch = (char)(entry >> ((charsPerEntry-1)*16));
1214                         entry <<= 16;
1215                         theString.append(ch);
1216                     }
1217                 }
1218             }
1219             if (entriesInCharSoFar > 0) {
1220                 while (entriesInCharSoFar < entriesPerChar) {
1221                     ch = (char)((int)ch >> bits);
1222                     ++entriesInCharSoFar;
1223                 }
1224                 theString.append(ch);
1225                 entriesInCharSoFar = 0;
1226             }
1227             result.append(Utility.formatForSource(theString.toString(), "    "));
1228             if (noConversion) {
1229                 result.append(").toCharArray()");
1230             }
1231             result.append(";\n\n  ");
1232 
1233             if (!noConversion) {
1234                 addInitializer(name, atype, entriesPerChar, bits, table.length);
1235             }
1236         }
1237         else {
1238             result.append("] = {");
1239             boolean castEntries = shiftEntries && (bits < 32);
1240             int printPerLine = hexFormat ? (bits == 1 ? 32*4 :
1241                 bits == 2 ? 16*4 :
1242                 bits == 4 ? 8*4 :
1243                 bits == 8 ? 8 :
1244                 bits == 16 ? 8 :
1245                 bits == 32 ? 4 : 2) :
1246                 (bits == 8 ? 8 :
1247                 bits == 16 ? 8 : 4);
1248             int printMask = properties ? 0 :
1249             Math.min(1 << size,
1250                 printPerLine >> (castEntries ? (Csyntax ? 2 : 1) : 0)) - 1;
1251             int commentShift = ((1 << size) == table.length) ? 0 : size;
1252             int commentMask = ((1 << size) == table.length) ? printMask : (1 << size) - 1;
1253             long val = 0;
1254             for (int j = 0; j < table.length; j++) {
1255                 if ((j & printMask) == 0) {
1256                     while (result.charAt(result.length() - 1) == ' ')
1257                         result.setLength(result.length() - 1);
1258                     result.append("\n    ");
1259                 }
1260         PRINT:  {
1261                 if (castEntries)
1262                     result.append("(").append(atype).append(")(");
1263                 long entry = table[j] >> extract;
1264                 int packMask = ((1 << (bits == 1 ? 5 : bits == 2 ? 4 : bits == 4 ? 3 : 2)) - 1);
1265                 int k = j & packMask;
1266                 if (bits >= 8)
1267                     val = entry;
1268                 else if (k == 0) {
1269                     val = entry;
1270                     break PRINT;
1271                 }
1272                 else {
1273                     val |= (entry << (k*bits));
1274                     if (k != packMask)
1275                         break PRINT;
1276                 }
1277                 if (val > maxPosEntry && !Csyntax) { // liu
1278                 // For values that are out of range, convert them to in-range negative values.
1279                 // Actually, output the '-' and convert them to the negative of the corresponding
1280                 // in-range negative values.  E.g., convert 130 == -126 (in 8 bits) -> 126.
1281                     result.append('-');
1282                     val = maxPosEntry + maxPosEntry + 2 - val;
1283                 }
1284                 if (hexFormat) {
1285                     result.append("0x");
1286                     if (bits == 8)
1287                         result.append(hex2((byte)val));
1288                     else if (bits == 16)
1289                         result.append(hex4((short)val));
1290                     else if (bits == 32 || bits < 8)
1291                         result.append(hex8((int)val));
1292                     else {
1293                         result.append(hex16((long)val));
1294                         if (!Csyntax)
1295                             result.append("L");
1296                     }
1297                 }
1298                 else {
1299                     if (bits == 8)
1300                         result.append(dec3(val));
1301                     else if (bits == 64) {
1302                         result.append(dec5(val));
1303                         if (!Csyntax)
1304                             result.append("L");
1305                     }
1306                     else
1307                         result.append(dec5(val));
1308                 }
1309                 if (shiftEntries)
1310                     result.append("<<").append(shift);
1311                 if (castEntries) result.append(")");
1312                 if (j < (table.length - 1))
1313                     result.append(", ");
1314                 else
1315                     result.append("  ");
1316                 if ((j & printMask) == printMask) {
1317                     result.append(" ").append(commentStart).append(" ");
1318                     if (hexComment)
1319                         result.append("0x").append(hex4((j & ~commentMask) << (16 - size)));
1320                     else
1321                         result.append(dec3((j & ~commentMask) >> commentShift));
1322                     if (properties) propertiesComments(result, val);
1323                     result.append(commentEnd);
1324                 }
1325                 } // end PRINT
1326             }
1327             result.append("\n  };\n\n  ");
1328         }
1329     }
1330 
1331     static void genCaseMapTableDeclaration(StringBuffer result) {
1332         String myTab = "    ";
1333         result.append(myTab + "static final char[][][] charMap;\n");
1334     }
1335 
1336     static void genCaseMapTable(StringBuffer result, SpecialCaseMap[] specialCaseMaps){
1337         String myTab = "    ";
1338         int ch;
1339         char[] map;
1340         result.append(myTab + "charMap = new char[][][] {\n");
1341         for (int x = 0; x < specialCaseMaps.length; x++) {
1342             ch = specialCaseMaps[x].getCharSource();
1343             map = specialCaseMaps[x].getUpperCaseMap();
1344             result.append(myTab + myTab);
1345             result.append("{ ");
1346             result.append("{\'\\u"+hex4(ch)+"\'}, {");
1347             for (int y = 0; y < map.length; y++) {
1348                 result.append("\'\\u"+hex4(map[y])+"\', ");
1349             }
1350             result.append("} },\n");
1351         }
1352         result.append(myTab + "};\n");
1353 
1354     }
1355 
1356     /**
1357     * The propertiesComments method generates comments describing encoded
1358     * character properties.
1359     *
1360     * @param result     a StringBuffer, to which the generated source code
1361     *                   text is to be appended
1362     * @param val                encoded character properties
1363     *
1364     * @see GenerateCharacter#genTable
1365     */
1366 
1367     static void propertiesComments(StringBuffer result, long val) {
1368         result.append("   ");
1369         switch ((int)(val & maskType)) {
1370             case UnicodeSpec.CONTROL:
1371                 result.append("Cc");
1372                 break;
1373             case UnicodeSpec.FORMAT:
1374                 result.append("Cf");
1375                 break;
1376             case UnicodeSpec.PRIVATE_USE:
1377                 result.append("Co");
1378                 break;
1379             case UnicodeSpec.SURROGATE:
1380                 result.append("Cs");
1381                 break;
1382             case UnicodeSpec.LOWERCASE_LETTER:
1383                 result.append("Ll");
1384                 break;
1385             case UnicodeSpec.MODIFIER_LETTER:
1386                 result.append("Lm");
1387                 break;
1388             case UnicodeSpec.OTHER_LETTER:
1389                 result.append("Lo");
1390                 break;
1391             case UnicodeSpec.TITLECASE_LETTER:
1392                 result.append("Lt");
1393                 break;
1394             case UnicodeSpec.UPPERCASE_LETTER:
1395                 result.append("Lu");
1396                 break;
1397             case UnicodeSpec.COMBINING_SPACING_MARK:
1398                 result.append("Mc");
1399                 break;
1400             case UnicodeSpec.ENCLOSING_MARK:
1401                 result.append("Me");
1402                 break;
1403             case UnicodeSpec.NON_SPACING_MARK:
1404                 result.append("Mn");
1405                 break;
1406             case UnicodeSpec.DECIMAL_DIGIT_NUMBER:
1407                 result.append("Nd");
1408                 break;
1409             case UnicodeSpec.LETTER_NUMBER:
1410                 result.append("Nl");
1411                 break;
1412             case UnicodeSpec.OTHER_NUMBER:
1413                 result.append("No");
1414                 break;
1415             case UnicodeSpec.CONNECTOR_PUNCTUATION:
1416                 result.append("Pc");
1417                 break;
1418             case UnicodeSpec.DASH_PUNCTUATION:
1419                 result.append("Pd");
1420                 break;
1421             case UnicodeSpec.END_PUNCTUATION:
1422                 result.append("Pe");
1423                 break;
1424             case UnicodeSpec.OTHER_PUNCTUATION:
1425                 result.append("Po");
1426                 break;
1427             case UnicodeSpec.START_PUNCTUATION:
1428                 result.append("Ps");
1429                 break;
1430             case UnicodeSpec.CURRENCY_SYMBOL:
1431                 result.append("Sc");
1432                 break;
1433             case UnicodeSpec.MODIFIER_SYMBOL:
1434                 result.append("Sk");
1435                 break;
1436             case UnicodeSpec.MATH_SYMBOL:
1437                 result.append("Sm");
1438                 break;
1439             case UnicodeSpec.OTHER_SYMBOL:
1440                 result.append("So");
1441                 break;
1442             case UnicodeSpec.LINE_SEPARATOR:
1443                 result.append("Zl"); break;
1444             case UnicodeSpec.PARAGRAPH_SEPARATOR:
1445                 result.append("Zp");
1446                 break;
1447             case UnicodeSpec.SPACE_SEPARATOR:
1448                 result.append("Zs");
1449                 break;
1450             case UnicodeSpec.UNASSIGNED:
1451                 result.append("unassigned");
1452                 break;
1453         }
1454 
1455         switch ((int)((val & maskBidi) >> shiftBidi)) {
1456             case UnicodeSpec.DIRECTIONALITY_LEFT_TO_RIGHT:
1457                 result.append(", L");
1458                 break;
1459             case UnicodeSpec.DIRECTIONALITY_RIGHT_TO_LEFT:
1460                 result.append(", R");
1461                 break;
1462             case UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER:
1463                 result.append(", EN");
1464                 break;
1465             case UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR:
1466                 result.append(", ES");
1467                 break;
1468             case UnicodeSpec.DIRECTIONALITY_EUROPEAN_NUMBER_TERMINATOR:
1469                 result.append(", ET");
1470                 break;
1471             case UnicodeSpec.DIRECTIONALITY_ARABIC_NUMBER:
1472                 result.append(", AN");
1473                 break;
1474             case UnicodeSpec.DIRECTIONALITY_COMMON_NUMBER_SEPARATOR:
1475                 result.append(", CS");
1476                 break;
1477             case UnicodeSpec.DIRECTIONALITY_PARAGRAPH_SEPARATOR:
1478                 result.append(", B");
1479                 break;
1480             case UnicodeSpec.DIRECTIONALITY_SEGMENT_SEPARATOR:
1481                 result.append(", S");
1482                 break;
1483             case UnicodeSpec.DIRECTIONALITY_WHITESPACE:
1484                 result.append(", WS");
1485                 break;
1486             case UnicodeSpec.DIRECTIONALITY_OTHER_NEUTRALS:
1487                 result.append(", ON");
1488                 break;
1489         }
1490         if ((val & maskUpperCase) != 0) {
1491             result.append(", hasUpper (subtract ");
1492             result.append((val & maskCaseOffset) >> shiftCaseOffset).append(")");
1493         }
1494         if ((val & maskLowerCase) != 0) {
1495             result.append(", hasLower (add ");
1496             result.append((val & maskCaseOffset) >> shiftCaseOffset).append(")");
1497         }
1498         if ((val & maskTitleCase) != 0) {
1499             result.append(", hasTitle");
1500         }
1501         if ((val & maskIdentifierInfo) == valueIgnorable) {
1502             result.append(", ignorable");
1503         }
1504         if ((val & maskIdentifierInfo) == valueJavaUnicodePart) {
1505             result.append(", identifier part");
1506         }
1507         if ((val & maskIdentifierInfo) == valueJavaStartUnicodePart) {
1508             result.append(", underscore");
1509         }
1510         if ((val & maskIdentifierInfo) == valueJavaWhitespace) {
1511             result.append(", whitespace");
1512         }
1513         if ((val & maskIdentifierInfo) == valueJavaOnlyStart) {
1514             result.append(", currency");
1515         }
1516         if ((val & maskIdentifierInfo) == valueJavaUnicodeStart) {
1517             result.append(", identifier start");
1518         }
1519         if ((val & maskNumericType) == valueDigit) {
1520             result.append(", decimal ");
1521             result.append((val & maskDigitOffset) >> shiftDigitOffset);
1522         }
1523         if ((val & maskNumericType) == valueStrangeNumeric) {
1524             result.append(", strange");
1525         }
1526         if ((val & maskNumericType) == valueJavaSupradecimal) {
1527             result.append(", supradecimal ");
1528             result.append((val & maskDigitOffset) >> shiftDigitOffset);
1529         }
1530     }
1531 
1532     static String[] tableNames = { "X", "Y", "Z", "P", "Q", "R", "S", "T", "U", "V", "W" };
1533 
1534     static String tableName(int j) { return tableNames[j]; }
1535 
1536     /**
1537     * The genAccess method generates source code for one table access expression.
1538     *
1539     * Most of the complexity stems from handling various options as to
1540     * table representation, such as whether it contains values so large that
1541     * they are represented as negative values and whether the table values are
1542     * preshifted.  This method also avoids such "ugly" expressions as shifting
1543     * by distance zero, masking when no masking is necessary, and so on.
1544     * For clarity, it generates expressions that do not rely on operator
1545     * precedence, but otherwise it avoids generating redundant parentheses.
1546     *
1547     * A generated expression might look like A[Y[(X[ch>>6]<<6)|(ch&0x3F)]]
1548     * or A[Z[Y[(X[ch>>7]<<4)|((ch>>3)&0xF)]|(ch&0x7)]], for example.
1549     *
1550     * @param tbl                the name of the final table to be accessed
1551     * @param var                the variable name that appeared in parentheses in the
1552     *                           "Lookup" command
1553     * @param bits       the number of bits (not bytes) to be used to represent
1554     *                   the final table entry
1555     * @return   the replacement text for the "Lookup(xxx)" command, as a String
1556     *
1557     * @see GenerateCharacter#replaceCommand
1558     */
1559 
1560     static String genAccess(String tbl, String var, int bits) {
1561         String access = null;
1562         int bitoffset = bits == 1 ? 5 : bits == 2 ? 4 : bits == 4 ? 3 : 0;
1563         for (int k = 0; k < sizes.length; k++) {
1564             int offset = ((k < sizes.length - 1) ? 0 : bitoffset);
1565             int shift = shifts[k] + offset;
1566             String shifted = (shift == 0) ? var : "(" + var + ">>" + shift + ")";
1567             int mask = (1 << (sizes[k] - offset)) - 1;
1568             String masked = (k == 0) ? shifted :
1569               "(" + shifted + "&0x" + hex(mask) + ")";
1570             String index = (k == 0) ? masked :
1571              (mask == 0) ? access : "(" + access + "|" + masked + ")";
1572             String indexNoParens = (index.charAt(0) != '(') ? index :
1573                  index.substring(1, index.length() - 1);
1574             String tblname = (k == sizes.length - 1) ? tbl : tableName(k);
1575             String fetched = tblname + "[" + indexNoParens + "]";
1576             String zeroextended = (zeroextend[k] == 0) ? fetched :
1577                 "(" + fetched + "&0x" + hex(zeroextend[k]) + ")";
1578             int adjustment = preshifted[k] ? 0 :
1579                sizes[k+1] - ((k == sizes.length - 2) ? bitoffset : 0);
1580             String adjusted = (preshifted[k] || adjustment == 0) ? zeroextended :
1581                 "(" + zeroextended + "<<" + adjustment + ")";
1582             String bitshift = (bits == 1) ? "(" + var + "&0x1F)" :
1583                 (bits == 2) ? "((" + var + "&0xF)<<1)" :
1584                 (bits == 4) ? "((" + var + "&7)<<2)" : null;
1585             String extracted = ((k < sizes.length - 1) || (bits >= 8)) ? adjusted :
1586                 "((" + adjusted + ">>" + bitshift + ")&" +
1587                 (bits == 4 ? "0xF" : "" + ((1 << bits) - 1)) + ")";
1588             access = extracted;
1589         }
1590         return access;
1591     }
1592 
1593     /* The command line arguments are decoded and used to set the following
1594      global variables.
1595      */
1596 
1597     static boolean verbose = false;
1598     static boolean nobidi = false;
1599     static boolean nomirror = false;
1600     static boolean identifiers = false;
1601     static boolean Csyntax = false;
1602     static String TemplateFileName = null;
1603     static String OutputFileName = null;
1604     static String UnicodeSpecFileName = null; // liu
1605     static String SpecialCasingFileName = null;
1606     static String PropListFileName = null;
1607     static boolean useCharForByte = false;
1608     static int[] sizes;
1609     static int bins = 0; // liu; if > 0, then perform search
1610     static boolean tableAsString = false;
1611     static boolean bLatin1 = false;
1612 
1613     static String commandLineDescription;
1614 
1615     /* Other global variables, equal in length to the "sizes" array. */
1616 
1617     static int[] shifts;
1618     static int[] zeroextend;
1619     static int[] bytes;
1620     static boolean[] preshifted;
1621     static long[][] tables;
1622 
1623 
1624     /* Other global variables */
1625     static String commentStart;
1626     static String commentEnd;
1627 
1628     static StringBuffer initializers = new StringBuffer();
1629 
1630     /* special casing rules for 1:M toUpperCase mappings */
1631     static SpecialCaseMap[] specialCaseMaps;
1632 
1633     /**
1634     * Process the command line arguments.
1635     *
1636     * The allowed flags in command line are:
1637     * <dl>
1638     * <dt> -verbose             <dd> Emit comments to standard output describing
1639     *                                   what's going on during the processing.
1640     * <dt> -nobidi              <dd> Do not include bidi categories in the
1641     *                                   encoded character properties.
1642     * <dt> -nomirror    <dd> Do no include mirror property in the encoded
1643     *                        character properties.
1644     * <dt> -identifiers         <dd> Generate tables for scanning identifiers only.
1645     * <dt> -c                   <dd> Output code in C syntax instead of Java syntax.
1646     * <dt> -o filename          <dd> Specify output file name.
1647     * <dt> -template filename   <dd> Specify template input file name.
1648     * <dt> -spec filename        <dd> Specify Unicode spec file name.
1649     * <dt> -specialcasing filename <dd> Specify Unicode special casing file name.
1650     * <dt> -search bins          <dd> Try different partitions into the specified
1651     *                                    number of bins.  E.g., for 2 bins, try
1652     *                                    16 0, 15 1,..., 0 16.
1653     * <dt> -string               <dd> Create table as string.  Only valid with Java
1654     *                                    syntax.
1655     * <dt> -latin1          <dd> Create a latin 1 only property table.
1656     * </dl>
1657     * In addition, decimal literals may appear as command line arguments;
1658     * each one represents the number of bits of the character to be broken
1659     * off at each lookup step.  If present, they must add up to 16 (the number
1660     * of bits in a char value).  For smaller tables, the last value should
1661     * be 0; values other than the last one may not be zero.  If no such
1662     * numeric values are provided, default values are used.
1663     *
1664     * @param args       the command line arguments, as an array of String
1665     *
1666     * @see GenerateCharacter#main
1667     */
1668 
1669     static void processArgs(String[] args) {
1670         StringBuffer desc = new StringBuffer("java GenerateCharacter");
1671         for (int j=0; j<args.length; ++j) {
1672             desc.append(" " + args[j]);
1673         }
1674         for (int j = 0; j < args.length; j++) {
1675             if (args[j].equals("-verbose") || args[j].equals("-v"))
1676                 verbose = true;
1677             else if (args[j].equals("-nobidi"))
1678                 nobidi = true;
1679             else if (args[j].equals("-nomirror"))
1680                 nomirror = true;
1681             else if (args[j].equals("-identifiers"))
1682                 identifiers = true;
1683             else if (args[j].equals("-c"))
1684                 Csyntax = true;
1685             else if (args[j].equals("-string"))
1686                 tableAsString = true;
1687             else if (args[j].equals("-o")) {
1688                 if (j == args.length - 1) {
1689                     FAIL("File name missing after -o");
1690                 }
1691                 else {
1692                     OutputFileName = args[++j];
1693                 }
1694             }
1695             else if (args[j].equals("-search")) {
1696                 if (j == args.length - 1)
1697                     FAIL("Bin count missing after -search");
1698                 else {
1699                     bins = Integer.parseInt(args[++j]);
1700                     if (bins < 1 || bins > 10)
1701                         FAIL("Bin count must be >= 1 and <= 10");
1702                 }
1703             }
1704             else if (args[j].equals("-template")) {
1705                 if (j == args.length - 1)
1706                     FAIL("File name missing after -template");
1707                 else
1708                     TemplateFileName = args[++j];
1709             }
1710             else if (args[j].equals("-spec")) { // liu
1711                 if (j == args.length - 1) {
1712                     FAIL("File name missing after -spec");
1713                 }
1714                 else {
1715                     UnicodeSpecFileName = args[++j];
1716                 }
1717             }
1718             else if (args[j].equals("-specialcasing")) {
1719                 if (j == args.length -1) {
1720                     FAIL("File name missing after -specialcasing");
1721                 }
1722                 else {
1723                     SpecialCasingFileName = args[++j];
1724                 }
1725             }
1726             else if (args[j].equals("-proplist")) {
1727                 if (j == args.length -1) {
1728                     FAIL("File name missing after -proplist");
1729                 }
1730                 else {
1731                     PropListFileName = args[++j];
1732                 }
1733             }
1734             else if (args[j].equals("-plane")) {
1735                 if (j == args.length -1) {
1736                     FAIL("Plane number missing after -plane");
1737                 }
1738                 else {
1739                     plane = Integer.parseInt(args[++j]);
1740                 }
1741                 if (plane > 0) {
1742                     bLatin1 = false;
1743                 }
1744             }
1745             else if ("-usecharforbyte".equals(args[j])) {
1746                 useCharForByte = true;
1747             }
1748             else if (args[j].equals("-latin1")) {
1749                 bLatin1 = true;
1750                 plane = 0;
1751             }
1752             else {
1753                 try {
1754                     int val = Integer.parseInt(args[j]);
1755                     if (val < 0 || val > 32) FAIL("Incorrect bit field width: " + args[j]);
1756                     if (sizes == null)
1757                         sizes = new int[1];
1758                     else {
1759                         int[] newsizes = new int[sizes.length + 1];
1760                         System.arraycopy(sizes, 0, newsizes, 0, sizes.length);
1761                         sizes = newsizes;
1762                     }
1763                     sizes[sizes.length - 1] = val;
1764                 }
1765                 catch(NumberFormatException e) {
1766                     FAIL("Unknown switch: " + args[j]);
1767                 }
1768             }
1769         }
1770         if (Csyntax && tableAsString) {
1771             FAIL("Can't specify table as string with C syntax");
1772         }
1773         if (sizes == null) {
1774             desc.append(" [");
1775             if (identifiers) {
1776                 int[] newsizes = { 8, 4, 4 };           // Good default values
1777                 desc.append("8 4 4]");
1778                 sizes = newsizes;
1779             }
1780             else {
1781                 int[] newsizes = { 10, 5, 1 }; // Guy's old defaults for 2.0.14: { 9, 4, 3, 0 }
1782                 desc.append("10 5 1]");
1783                 sizes = newsizes;
1784             }
1785         }
1786         if (UnicodeSpecFileName == null) { // liu
1787             UnicodeSpecFileName = DefaultUnicodeSpecFileName;
1788             desc.append(" [-spec " + UnicodeSpecFileName + ']');
1789         }
1790         if (SpecialCasingFileName == null) {
1791             SpecialCasingFileName = DefaultSpecialCasingFileName;
1792             desc.append(" [-specialcasing " + SpecialCasingFileName + ']');
1793         }
1794         if (PropListFileName == null) {
1795             PropListFileName = DefaultPropListFileName;
1796             desc.append(" [-proplist " + PropListFileName + ']');
1797         }
1798         if (TemplateFileName == null) {
1799             TemplateFileName = (Csyntax ? DefaultCTemplateFileName
1800                   : DefaultJavaTemplateFileName);
1801             desc.append(" [-template " + TemplateFileName + ']');
1802         }
1803         if (OutputFileName == null) {
1804             OutputFileName = (Csyntax ? DefaultCOutputFileName
1805                     : DefaultJavaOutputFileName);
1806             desc.append(" [-o " + OutputFileName + ']');
1807         }
1808         commentStart = (Csyntax ? "/*" : "//");
1809         commentEnd = (Csyntax ? " */" : "");
1810         commandLineDescription = desc.toString();
1811     }
1812 
1813     private static void searchBins(long[] map, int binsOccupied) throws Exception {
1814         int bitsFree = 16;
1815         for (int i=0; i<binsOccupied; ++i) bitsFree -= sizes[i];
1816         if (binsOccupied == (bins-1)) {
1817             sizes[binsOccupied] = bitsFree;
1818             generateForSizes(map);
1819         }
1820         else {
1821             for (int i=1; i<bitsFree; ++i) { // Don't allow bins of 0 except for last one
1822                 sizes[binsOccupied] = i;
1823                 searchBins(map, binsOccupied+1);
1824             }
1825         }
1826     }
1827 
1828     private static void generateForSizes(long[] map) throws Exception {
1829         int sum = 0;
1830         shifts = new int[sizes.length];
1831         for (int k = sizes.length - 1; k >= 0; k--) {
1832             shifts[k] = sum;
1833             sum += sizes[k];
1834         }
1835         if ((1 << sum) < map.length || (1 << (sum - 1)) >= map.length) {
1836             FAIL("Bit field widths total to " + sum +
1837              ": wrong total for map of size " + map.length);
1838         }
1839         // need a table for each set of lookup bits in char
1840         tables = new long[sizes.length][];
1841         // the last table is the map
1842         tables[sizes.length - 1] = map;
1843         for (int j = sizes.length - 1; j > 0; j--) {
1844             if (verbose && bins==0)
1845                 System.err.println("Building map " + (j+1) + " of bit width " + sizes[j]);
1846             long[][] temp = buildTable(tables[j], sizes[j]);
1847             tables[j-1] = temp[0];
1848             tables[j] = temp[1];
1849         }
1850         preshifted = new boolean[sizes.length];
1851         zeroextend = new int[sizes.length];
1852         bytes = new int[sizes.length];
1853         for (int j = 0; j < sizes.length - 1; j++) {
1854             int len = tables[j+1].length;
1855             int size = sizes[j+1];
1856             if (len > 0x100 && (len >> size) <= 0x100) {
1857                 len >>= size;
1858                 preshifted[j] = false;
1859             }
1860             else if (len > 0x10000 && (len >> size) <= 0x10000) {
1861                 len >>= size;
1862                 preshifted[j] = false;
1863             }
1864             else preshifted[j] = true;
1865             if (Csyntax)
1866                 zeroextend[j] = 0;
1867             else if (len > 0x7F && len <= 0xFF) {
1868                 if (!useCharForByte) {
1869                     zeroextend[j] = 0xFF;
1870                 }
1871             } else if (len > 0x7FFF && len <= 0xFFFF)
1872                 zeroextend[j] = 0xFFFF;
1873             else zeroextend[j] = 0;
1874             if (len <= 0x100) bytes[j] = 1;
1875             else if (len <= 0x10000) bytes[j] = 2;
1876             else bytes[j] = 4;
1877         }
1878         preshifted[sizes.length - 1] = true;
1879         zeroextend[sizes.length - 1] = 0;
1880         bytes[sizes.length - 1] = 0;
1881         if (bins > 0) {
1882             int totalBytes = getTotalBytes();
1883             String access = genAccess("A", "ch", (identifiers ? 2 : 32));
1884             int accessComplexity = 0;
1885             for (int j=0; j<access.length(); ++j) {
1886                 char ch = access.charAt(j);
1887                 if ("[&|><".indexOf(ch) >= 0) ++accessComplexity;
1888                 if (ch == '<' || ch == '>') ++j;
1889             }
1890             System.out.print("(");
1891             for (int j=0; j<sizes.length; ++j) System.out.print(" " + sizes[j]);
1892             System.out.println(" ) " + totalBytes + " " + accessComplexity + " " + access);
1893             return;
1894         }
1895         if (verbose) {
1896             System.out.println("    n\t size\tlength\tshift\tzeroext\tbytes\tpreshifted");
1897             for (int j = 0; j < sizes.length; j++) {
1898                 System.out.println(dec5(j) + "\t" +
1899                     dec5(sizes[j]) + "\t" +
1900                     dec5(tables[j].length) + "\t" +
1901                     dec5(shifts[j]) + "\t" +
1902                     dec5(zeroextend[j]) + "\t" +
1903                     dec5(bytes[j]) + "\t " +
1904                     preshifted[j]);
1905             }
1906         }
1907         if (verbose) {
1908             System.out.println("Generating source code for class Character");
1909             System.out.println("A table access looks like " +
1910                          genAccess("A", "ch", (identifiers ? 2 : 32)));
1911         }
1912         generateCharacterClass(TemplateFileName, OutputFileName);
1913     }
1914 
1915     /**
1916     * The main program for generating source code for the Character class.
1917     * The basic outline of its operation is:
1918     * <ol>
1919     * <li> Process the command line arguments.  One result of this process
1920     *           is a list of sizes (measured in bits and summing to 16).
1921     * <li> Get the Unicode character property data from the specification file.
1922     * <li> From that, build a map that has, for each character code, its
1923     *           relevant properties encoded as a long integer value.
1924     * <li> Repeatedly compress the map, producing a compressed table and a
1925     *           new map.  This is done once for each size value in the list.
1926     *           When this is done, we have a set of tables.
1927     * <li> Make some decisions about table representation; record these
1928     *           decisions in arrays named preshifted, zeroextend, and bytes.
1929     * <li> Generate the source code for the class Character by performing
1930     *           macro processing on a template file.
1931     * </ol>
1932     *
1933     * @param args       the command line arguments, as an array of String
1934     *
1935     * @see GenerateCharacter#processArgs
1936     * @see UnicodeSpec@readSpecFile
1937     * @see GenerateCharacter#buildMap
1938     * @see GenerateCharacter#buildTable
1939     * @see GenerateCharacter#generateCharacterClass
1940     */
1941 
1942     public static void main(String[] args) {
1943         processArgs(args);
1944         try {
1945 
1946             UnicodeSpec[] data = UnicodeSpec.readSpecFile(new File(UnicodeSpecFileName), plane);
1947             specialCaseMaps = SpecialCaseMap.readSpecFile(new File(SpecialCasingFileName), plane);
1948             PropList propList = PropList.readSpecFile(new File(PropListFileName), plane);
1949 
1950             if (verbose) {
1951                 System.out.println(data.length + " items read from Unicode spec file " + UnicodeSpecFileName); // liu
1952             }
1953             long[] map = buildMap(data, specialCaseMaps, propList);
1954             if (verbose) {
1955                 System.err.println("Completed building of initial map");
1956             }
1957 
1958             if (bins == 0) {
1959                 generateForSizes(map);
1960             }
1961             else {
1962                 while (bins > 0) {
1963                     sizes = new int[bins];
1964                     searchBins(map, 0);
1965                     --bins;
1966                 }
1967             }
1968             if (verbose && false) {
1969                 System.out.println("Offset range seen: -" + hex8(-minOffsetSeen) + "..+" +
1970                              hex8(maxOffsetSeen));
1971                 System.out.println("          allowed: -" + hex8(-minOffset) + "..+" +
1972                              hex8(maxOffset));
1973             }
1974         }
1975         catch (FileNotFoundException e) { FAIL(e.toString()); }
1976         catch (IOException e) { FAIL(e.toString()); }
1977         catch (Throwable e) {
1978             System.out.println("Unexpected exception:");
1979             e.printStackTrace();
1980             FAIL("Unexpected exception!");
1981         }
1982         if (verbose) { System.out.println("Done!");}
1983     }
1984 
1985 }   // end class